{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理PDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PDF切分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 手动切分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以使用 PyMuPDF、PyPDF 等工具读取文件，然后使用 nltk 等工具对文档进行切分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "安装相关包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install PyMuPDF==1.24.14 nltk==3.9.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 下载 punkt 句子分割器\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('punkt_tab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fitz  # PyMuPDF\n",
    "import nltk\n",
    "\n",
    "\n",
    "def extract_text_from_pdf(pdf_path):\n",
    "    # 打开 PDF 文件\n",
    "    doc = fitz.open(pdf_path)\n",
    "    # 拼接句子并返回\n",
    "    return \"\".join([page.get_text() for page in doc])\n",
    "\n",
    "\n",
    "def split_text_into_sentences(text, min_len=5):\n",
    "    # 使用 nltk 进行句子分割\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    # 过滤掉太短的句子\n",
    "    sentences = list(filter(lambda x: len(x.split(' ')) > min_len, sentences))\n",
    "    return sentences\n",
    "\n",
    "\n",
    "pdf_path = '../datas/papers/reaction-predict-optim/s41467-020-19267-x.pdf'\n",
    "text = extract_text_from_pdf(pdf_path)\n",
    "sentences = split_text_into_sentences(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 使用 langchain 切分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Loader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[How to load PDFs](https://python.langchain.com/docs/how_to/document_loader_pdf/)\n",
    "\n",
    "[PDF Loaders](https://python.langchain.com/docs/integrations/document_loaders/#pdfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "loader = PyPDFLoader(\n",
    "    \"../datas/papers/reaction-predict-optim/s41467-020-19267-x.pdf\",\n",
    ")\n",
    "doc = loader.load()\n",
    "doc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyMuPDFLoader\n",
    "loader = PyMuPDFLoader(\"../datas/papers/reaction-predict-optim/s41467-020-19267-x.pdf\")\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Splitter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Text splitters](https://python.langchain.com/docs/concepts/text_splitters/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    # Set a really small chunk size, just to show.\n",
    "    chunk_size=500,\n",
    "    chunk_overlap=0,\n",
    "    separators=[\n",
    "        \"\\n\\n\",\n",
    "        \"\\n\",\n",
    "        \" \",\n",
    "        \".\",\n",
    "        \",\",\n",
    "        \"\\u200b\",  # Zero-width space\n",
    "        \"\\uff0c\",  # Fullwidth comma\n",
    "        \"\\u3001\",  # Ideographic comma\n",
    "        \"\\uff0e\",  # Fullwidth full stop\n",
    "        \"\\u3002\",  # Ideographic full stop\n",
    "        \"\",\n",
    "    ],\n",
    ")\n",
    "\n",
    "splitted_doc = text_splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in splitted_doc[:5]:\n",
    "    print(\"+++++++++++++++++++++++++++ \", len(doc.page_content))\n",
    "    print(doc.page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_text_splitters import SentenceTransformersTokenTextSplitter\n",
    "\n",
    "splitter = SentenceTransformersTokenTextSplitter(chunk_overlap=100)\n",
    "splitted_doc = splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in splitted_doc[:5]:\n",
    "    print(\"+++++++++++++++++++++++++++ \", len(doc.page_content))\n",
    "    print(doc.page_content)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
